# Moths - an environment for mathy calculations
# Copyright (C) 2006 Mike Sutjipto
# Licensed under the GPL

require 'strscan'

module Moths
  
  # The tokenizer - essentially an iterator that takes in a string, and
  # returns the tokens relevant to moths
  #
  class Tokenizer
    def initialize(string)
      @s = StringScanner.new(string.chomp)
    end

    # Puts back the last token
    #
    def put_back
      @s.unscan
    end

    # True if there are more tokens left
    #
    def next?
      !@s.eos?
    end

    # Returns the next token
    #
    def next
      # get rid of white space
      # TODO get rid of other types of whitespace
      @s.scan(/\s+/) 

      if scanned = @s.scan(/~?\d+\.\d*/) then
        token_type = :realNumber
        scanned = scanned.sub(/~/, '-')
      elsif scanned = @s.scan(/~?\d+/) then
        token_type = :integer
        scanned = scanned.sub(/~/, '-')
      elsif scanned = @s.scan(/,/) then
        token_type = :comma
      elsif scanned = @s.scan(/\(/) then
        token_type = :openParens
      elsif scanned = @s.scan(/\)/) then
        token_type = :closeParens
      elsif scanned = @s.scan(/\[/) then
        token_type = :openBracket
      elsif scanned = @s.scan(/\]/) then
        token_type = :closeBracket
      elsif scanne = @s.scan(/\\/) then 
        token_type = :backslash
      elsif scanned = @s.scan(/=/) then
        token_type = :equals
      elsif scanned = @s.scan(/-/) then
        token_type = :minus
      elsif scanned = @s.scan(/\+/) then
        token_type = :plus
      elsif scanned = @s.scan(/\*/) then
        token_type = :times
      elsif scanned = @s.scan(/\//) then
        token_type = :div
      elsif scanned = @s.scan(/\^/) then
        token_type = :exp
      elsif scanned = @s.scan(/\bdef\b/) then
        token_type = :definition
      elsif scanned = @s.scan(/\bed\b/) then
        token_type = :edit
      elsif scanned = @s.scan(/\w+/) then 
        token_type = :identifier
      else
        raise MothsError, 'Unknown Token!'
      end

      return Token.new(token_type, scanned)
    end
  end
  
end
